# -*- coding: utf-8 -*-
import scrapy
from scrapy.http.response.html import HtmlResponse
from scrapy.selector.unified import Selector
from scrapy import Request
from ..items import AnalysisItem
import re


class HomeworkSpider(scrapy.Spider):
    name = 'homework'
    allowed_domains = ['gitee.com']
    start_urls = ['https://gitee.com/magedu/python28']
    # regx = re.compile("P\w*-(\w*)")

    def parse(self, response):
        if response.url == self.start_urls[0]:
            week_folders = response.xpath("//div[@class='five wide column tree-item-file-name tree-folder tree-list-item']/a")
            for folder_url in week_folders:
                # print(type(f.re('href=".*?"')))
                # print(folder_url.css("a::attr(href)").re("/\w*/\w*/\w*/\w*/.*?[^/]*"), folder_url.css("a::attr(title)").get())

                new_url = folder_url.css("a::attr(href)").re("/\w*/\w*/\w*/\w*/.*?[^/]*")
                # print(new_url)
                yield Request(response.urljoin(new_url[0]))

        if response.url != self.start_urls[0]:
            item = AnalysisItem()
            item['week'] = response.xpath("//div[@id='path-breadcrumb']/strong/text()").extract_first().strip()
            selectors = response.xpath("//div[@class='five wide column tree-item-file-name tree-folder "
                                       "tree-list-item']/a/text()").extract()

            for selector in selectors:
                item['classmate_name'] = selector.strip()
                yield item